package com.github.rabbithome;


import com.github.rabbithome.pipline.GanjiCSVPipeline;
import com.github.rabbithome.processor.GanjiPageProcessor;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.Pipeline;

import java.io.BufferedWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;

/**
 * Created by tomalloc on 17-1-6.
 */
public class GanjiBootstrap {

    public static void main(String[] args) throws IOException {
        String userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36";
        String refer = "http://bj.ganji.com/fang1/chaoyang/a1b1500e3500h1m1o11/";

//        String url="http://bj.ganji.com/fang1/haidian/a1b2000e3500/";
        //顺义
//        String url="http://bj.ganji.com/fang1/shunyi/a1b1500e3500m1/";
        //丰台
//        String url="http://bj.ganji.com/fang1/fengtai/a1b1500e3500m1/";
        //大兴
//        String url="http://bj.ganji.com/fang1/daxing/a1b2000e3500/";

        //朝阳
        String url="http://bj.ganji.com/fang1/chaoyang/a1b1500e3500m1/";

        Site site = Site.me().setUserAgent(userAgent).addHeader("Referer", refer).setRetryTimes(3).setSleepTime(2000);
        BufferedWriter bufferedWriter=Files.newBufferedWriter(Paths.get("/home/tomalloc/房/ganji-朝阳.1.7.csv"),StandardOpenOption.CREATE);
        Pipeline pipeline=new GanjiCSVPipeline(bufferedWriter);
        Spider.create(new GanjiPageProcessor(site)).addUrl(url).addPipeline(pipeline).thread(5).run();
        bufferedWriter.close();
    }
}
